﻿using Crawler.DataHelper;
using Crawler.DBEntities;
using Crawler.WebEntities;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;

namespace Crawler
{
    /// <summary>
    /// MainWindow.xaml 的交互逻辑
    /// </summary>
    public partial class MainWindow : Window
    {
        string _contentRegex;
        //种子是否有可以匹配帖子内容的表达式
        Boolean _State = false;
        //控制线程运行与否的变量
        bool _ThreadsRunning;
        // unique Uri's queue
        private Queue _queueURLS = new Queue ();
        // threads array    线程列表
        private Thread[] _threadsRun = new Thread[10];
        // thread that take the browse editor text to parse it
        private Thread _threadParse;
        // binary tree to keep unique Uri's
        private SortTree _urlStorage;
        private string[] strExcludeHosts;
        private string[] ExcludeHosts
        {
            get { return strExcludeHosts; }
            set { strExcludeHosts = value; }
        }
        //设置一个爬虫抓取前的等待时间
        private int nSleepConnectTime;
        private int SleepConnectTime
        {
            get { return nSleepConnectTime; }
            set { nSleepConnectTime = value; }
        }
        //当队列为空的时候的等待时间
        private int nSleepFetchTime;
        private int SleepFetchTime
        {
            get { return nSleepFetchTime; }
            set { nSleepFetchTime = value; }
        }
        public MainWindow()
        {
            InitializeComponent();
        }
        private void Button_StartClick(object sender, RoutedEventArgs e)
        {
            VisitedHelper.DelAllVisited();
            if (this.txb_seedUri.Text != null)
            {
                SleepConnectTime = 1;
                SleepFetchTime = 1;
                this.btn_start.IsEnabled = false;
                _ThreadsRunning = true;
                MyUri tempUri = new MyUri(this.txb_seedUri.Text);
                seedUri = tempUri;
                this.EnqueueUri(seedUri,false);
                CreatThreads();
            }
            
        }
        /// <summary>
        /// 创建线程
        /// </summary>
        public void CreatThreads()
        {
            for (int nIndex = 0; nIndex < 1; nIndex++)
            {
                // check if thread not created or not suspended
                //判断线程是否没有创建或者不是中止了
                if (_threadsRun[nIndex] == null || _threadsRun[nIndex].ThreadState != ThreadState.Suspended)
                {
                    //创建一个线程
                    _threadsRun[nIndex] = new Thread(new ThreadStart(ThreadRunFunction));
                    //用序号做线程的名字
                    _threadsRun[nIndex].Name = nIndex.ToString();
                    //启动线程
                    _threadsRun[nIndex].Start();
                }
                //判断线程是否被中止
                else if (_threadsRun[nIndex].ThreadState == ThreadState.Suspended)
                {
                    _threadsRun[nIndex].Resume();
                }
            }
        }
        
        /// <summary>
        /// 线程运行时执行的函数
        /// </summary>
        void ThreadRunFunction()
        {
            while (_ThreadsRunning && int.Parse(Thread.CurrentThread.Name) < 1)
            {
                MyUri uri = DequeueUri();
                if (uri != null)
                {
                    if (SleepConnectTime > 0)
                        Thread.Sleep(SleepConnectTime * 1000);
                    ParseUri(uri);
                }
                else
                    Thread.Sleep(SleepFetchTime * 1000);
            }
        }
        /// <summary>
        /// 从队列中出一个uri
        /// </summary>
        /// <returns></returns>

        MyUri DequeueUri()
        {
            Monitor.Enter(_queueURLS);
            MyUri uri = null;
            try
            {
                uri = (MyUri)_queueURLS.Dequeue();
            }
            catch (Exception)
            {
            }
            Monitor.Exit(_queueURLS);
            return uri;
        }
        //种子uri的string
        private MyUri seedUri;

        public MyUri SeedUri
        {
            get { return seedUri; }
            set { seedUri = value; }
        }
        /// <summary>
        /// 将获取的网页转换成string并返回
        /// </summary>
        /// <param name="bParse"></param>
        /// <param name="response"></param>
        /// <param name="itemLog"></param>
        /// <returns></returns>
        private string ReceiveString(MyUri uri)
        {
            string strResponse = null;
            strResponse = WebHelper.GetContent(uri);
            return strResponse;
        }
        /// <summary>
        /// 处理获取到的链接
        /// </summary>
        /// <param name="uri"></param>
        /// <returns>返回-2：解析过程出错
        ///          返回-1：解析的uri是其他地址；
        ///          返回0：该uri已被解析过；
        ///          返回1：解析的Uri是种子；
        ///          返回2：解析的uri是板块；
        ///          返回3：解析的uri是帖子</returns>
        private int HandleUri(ref MyUri uri, string strSponse)
        {
            bool isHandleSucceed = false;
            if (IsVisited(uri))
                return 0;
            else
                if (IsSeedUri(uri))                                         //是种子uri
                {
                    isHandleSucceed = HandleSeedUri(ref uri, strSponse);
                    if (isHandleSucceed)
                        return 1;
                    else
                        return -2;
                }
                else
                    if (IsPostUri(strSponse))                              //是帖子uri
                        {
                            isHandleSucceed = HandlePostUri(uri, strSponse);
                            if (isHandleSucceed)
                                return 3;
                            else
                                return -2;
                        }
                    
                    else
                        if (IsForumUri(strSponse))                          //是板块uri
                            {
                                isHandleSucceed = HandleForumUri(ref uri, strSponse);
                                if (uri.Id != 0)
                                    return 2;
                                else
                                    return -2;
                            }
                        else
                            return -1;
        }
        /// <summary>
        /// 处理帖子uri
        /// </summary>
        /// <param name="uri"></param>
        /// <param name="strSponse"></param>
        private bool HandlePostUri(MyUri uri, string strSponse)
        {
            bool isHandled = false;
            if (IsExistPost(uri))     //表中存在该帖子，不继续操作
                isHandled = true;
            else                      //表中不存在该帖子，添加
            {
                Post post = new Post();
                SetPostInfo(strSponse, ref post, uri);
                if (uri.CanSave == true&&uri.Fk_forum!=0&&uri.Fk_seed!=0)             
                {
                    isHandled = InsertToDB_post(post, ref uri);
                    InsertToDB_visited(uri);
                }
            }
            return isHandled;
        }
        /// <summary>
        /// 添加访问过的uri到visited表
        /// </summary>
        /// <param name="uri"></param>
        private void InsertToDB_visited(MyUri uri)
        {
            VisitedHelper.InsertToVisited(uri);
        }
        /// <summary>
        /// 处理板块uri，成功则返回true
        /// </summary>
        /// <param name="uri"></param>
        /// <param name="strSponse"></param>
        private bool HandleForumUri(ref MyUri uri, string strSponse)
        {
            bool isHandled =true;
            Forum forum = new Forum();
            SetFormInfo(uri, ref forum);
            if (IsExistForum(uri))        //表中存在板块信息，更新
            {
                if (uri.CanSave == true)                //如果种子的帖子内容获取正则已确定
                {
                    isHandled = UpdateToDB_forum(forum, ref uri);
                }
            }
            else                          //表中不存在板块信息，添加
            {
                
                if (uri.CanSave == true)
                {
                    isHandled = InsertToDB_forum(forum, ref uri);
                }
            }
            InsertToDB_visited(uri);
            return isHandled;
        }
        /// <summary>
        /// 处理种子uri
        /// </summary>
        /// <param name="uri"></param>
        /// <param name="strSponse"></param>
        /// <returns></returns>
        private bool HandleSeedUri(ref MyUri uri, string strSponse)
        {
            bool isHandled = false;
            if (IsExistSeed(uri))                                  //种子Uri是否在表中存在
            {
                _State = true;         
                uri.CanSave = true;
                _contentRegex = SeedHelper.GetPostRegex(uri);     //获取匹配帖子内容的正则表达式
                if (IsSeedTimeOut(uri))                            //种子Uri是否已过期，已过期则添加
                {
                    Seed seed = new Seed();
                    GetSeedInfo(ref seed, uri);
                    isHandled = UpdateToDB_seed(seed, ref uri);             //获取种子id并赋给seedUri addadd
                }
                else
                    isHandled = EnqueueForumUries(uri);  //未过期，把种子uri相关的板块uri放到队列中
                 
            }
                                        
            else
            {
                _State = false;
                seedUri.CanSave = false;
            }
            InsertToDB_visited(uri);
            return isHandled;
        }

        //判断是否从种子开始解析
        bool isStartFromSeed = true;
        /// <summary>
        /// 将种子uri相关的板块uri放入队列，设置IsStartFromSeed为false
        /// </summary>
        /// <param name="uri"></param>
        private bool EnqueueForumUries(MyUri uri)
        {
            isStartFromSeed = false;
            List<MyUri> uris = SeedHelper.GetForumUries(uri);
            if (uris.Count != 0)
            {
                foreach (var auri in uris)
                {
                    auri.CanSave = uri.CanSave;
                    EnqueueUri(auri, false);
                }
                return true;
            }
            else
                return false;
        }
        // push uri to the queue
        /// <summary>
        /// 将新的Uri放入队列
        /// </summary>
        /// <param name="uri"></param>
        /// <param name="bCheckRepetition"></param>
        /// <returns></returns>

        bool EnqueueUri(MyUri uri, bool bCheckRepetition)
        {
            // add the uri to the binary tree to check if it is duplicated or not
            //if (bCheckRepetition == true && AddURL(ref uri) == false)                           //排除无效的地址
            //    return false;

            Monitor.Enter(_queueURLS);
            try
            {
                // add the uri to the queue
                _queueURLS.Enqueue(uri);
            }
            catch (Exception)
            {
            }
            Monitor.Exit(_queueURLS);

            return true;
        }
        /// <summary>
        /// 把uri的字符串保存到树中
        /// </summary>
        /// <param name="uri"></param>
        /// <returns></returns>
        bool AddURL(ref MyUri uri)
        {
            foreach (string str in ExcludeHosts)
                if (str.Trim().Length > 0 && uri.Host.ToLower().IndexOf(str.Trim()) != -1)
                {
                    //LogError(uri.AbsoluteUri, "\r\nHost excluded as it includes reserved pattern (" + str + ")");
                    return false;
                }
            Monitor.Enter(_urlStorage);
            bool bNew = false;
            try
            {   //把uri的字符串保存到树中
                string strURL = uri.AbsoluteUri;
                bNew = _urlStorage.Add(ref strURL).Count == 1;
            }
            catch (Exception)
            {
            }
            Monitor.Exit(_urlStorage);

            return bNew;
        }
        /// <summary>
        /// 更新板块的信息到数据库
        /// </summary>
        /// <param name="forum"></param>
        /// <param name="uri"></param>
        private bool UpdateToDB_forum(Forum forum, ref MyUri uri)
        {
            return ForumHelper.UpdateForum(ref uri, forum);
        }
        /// <summary>
        /// 更新数据库中该种子的信息,成功返回true
        /// </summary>
        /// <param name="seed"></param>
        /// <param name="uri"></param>
        private bool UpdateToDB_seed(Seed seed, ref MyUri uri)
        {
            return SeedHelper.UpdateToDB_seed(seed, ref uri);
        }
        /// <summary>
        /// 保存种子页的信息到数据库，并返回该记录的id作为uri的id， 成功返回true   addadd
        /// </summary>
        /// <param name="seed"></param>
        /// <param name="uri"></param>
        private bool AddToDB_seed(ref MyUri uri, string contentRegex)
        {
            return SeedHelper.InsertSeed(ref uri, contentRegex);
        }
        /// <summary>
        /// 获取种子uri的信息并且赋值给seed
        /// </summary>
        /// <param name="strSponse"></param>
        private void GetSeedInfo(ref Seed seed, MyUri uri)
        {
            SeedHelper.GetSeedInfo(ref seed, uri);
        }
        /// <summary>
        /// 判断是否是种子uri
        /// </summary>
        /// <returns></returns>
        private bool IsSeedUri(MyUri uri)
        {
            if (uri.AbsoluteUri.Equals(seedUri.AbsoluteUri))
                return true;
            else
                return false;
        }
        /// <summary>
        /// 是否是已访问过的uri
        /// </summary>
        /// <param name="uri"></param>
        /// <returns></returns>
        private bool IsVisited(MyUri uri)
        {
            return VisitedHelper.IsVisited(uri);
        }
        /// <summary>
        /// 保存帖子的信息到数据库,外键不全的不添加，返回记录的id设置uri的id,成功则返回true
        /// </summary>
        /// <param name="post"></param>
        private bool InsertToDB_post(Post post, ref MyUri uri)
        {
            return PostHelper.InsertPostToDB(ref post);
        }
        /// <summary>
        /// 获取帖子的信息并放在post中
        /// </summary>
        /// <param name="newUri"></param>
        /// <param name="post"></param>
        private bool SetPostInfo(String strSponse, ref Post post, MyUri uri)
        {
            if (_State == false)
            {
                string contentRegex = PostHelper.GetContentRegex(strSponse);
                if (contentRegex.Equals("没有匹配的正则表达式"))
                    MessageBox.Show("没有匹配帖子内容的正则表达式");
                else
                {
                    _contentRegex = contentRegex;
                    _State = true;
                    AddToDB_seed(ref seedUri,contentRegex);
                    _queueURLS.Clear();
                    seedUri.CanSave = true;
                    _queueURLS.Enqueue(seedUri);
                }
                return false;
            }
            else
            {
                    post.Passage = PostHelper.GetPostContent(strSponse, _contentRegex);           
                    post.Title = PostHelper.GetPostTitle(strSponse);
                    post.Date = PostHelper.GetPostTime(strSponse);
                    post.Url = uri.AbsoluteUri;
                return true;
            }
            
        }
        /// <summary>
        /// 是否在数据库中已存在
        /// </summary>
        /// <param name="newUri"></param>
        /// <returns></returns>
        private bool IsExistPost(MyUri newUri)
        {
            MysqlHelper helper = new MysqlHelper();
            return helper.IsPostExist(newUri.AbsoluteUri);
        }
        /// <summary>
        /// 是否是帖子uri
        /// </summary>
        /// <param name="newUri"></param>
        /// <returns></returns>
        private bool IsPostUri(string response)
        {
            if (RegexHelper.HasBoultSign(response)|RegexHelper.HasDaYuHao(response)|RegexHelper.HasEmLabel(response))
            {
                if (RegexHelper.HasSofa(response))
                    return true;
                else
                    if (RegexHelper.HasDayuhao_zhengwen(response))
                        return true;
                    else
                        return false;
            }
            else
                return false;
        }
        /// <summary>
        /// 将板块的信息存入数据库,返回记录的id设置uri的id,成功返回true
        /// </summary>
        /// <param name="forum"></param>
        private bool InsertToDB_forum(Forum forum, ref MyUri uri)
        {
            return ForumHelper.InsertForum(ref uri, forum);
        }
        /// <summary>
        /// 获取板块信息并赋给forum
        /// </summary>
        /// <param name="newUri"></param>
        /// <param name="forum"></param>
        private void SetFormInfo(MyUri uri, ref Forum forum)
        {
            ForumHelper.SetForumInfo(uri, ref forum);
        }
        /// <summary>
        /// 判断是否已在表中存在
        /// </summary>
        /// <param name="newUri"></param>
        /// <returns></returns>
        private bool IsExistForum(MyUri uri)
        {
            return ForumHelper.IsFromExist(uri);
        }
        /// <summary>
        /// 判断是否是板块uri
        /// </summary>
        /// <param name="uri"></param>
        /// <returns></returns>
        private bool IsForumUri(string strResponse)
        {
            bool hasDaYuHao = false;
            bool hasEm = false;
            bool isForumUri = false;
            bool hasBoultSign = false;

            string dayuhaoRef = RegexHelper.regexDaYuHao;
            string emRef = RegexHelper.regexEmLabel;
            string boultSignRef = RegexHelper.regexBoultSign;
            string creamRef = RegexHelper.regexCream;

            hasDaYuHao = new Regex(dayuhaoRef).IsMatch(strResponse);
            hasEm = new Regex(emRef).IsMatch(strResponse);
            hasBoultSign = new Regex(boultSignRef).IsMatch(strResponse);
            if (hasDaYuHao == true || hasEm == true ||hasBoultSign == true)                                  //要不要加箭头判断？addadd
                isForumUri = new Regex(creamRef).IsMatch(strResponse);
            return isForumUri;
            //return false;
        }
        /// <summary>
        /// 解析uri
        /// </summary>
        /// <param name="uri"></param>
        /// <param name="request"></param>
        void ParseUri(MyUri uri)
        {
            int uriStyle;
            string strStatus = "";

            try
            {
                bool bParse = true;
                string strResponse = "";
                strResponse = ReceiveString(uri);
                uriStyle = HandleUri(ref uri, strResponse);                //处理了uri并返回uri的类型

                //if (ThreadsRunning == true && bParse == true && uri.Depth < WebDepth)             // 要修改
                //{
                if (true)
                {
                    string temp;
                    string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']";
                    MatchCollection matches = GetRefs(strResponse, ref strStatus);
                    List<string> strs = new List<string>();
                    foreach (Match match in matches)
                    {
                        strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
                        try
                        {
                            
                            MyUri newUri = GetNewUri(strRef, uri);
                            SetUriCanSave(ref newUri);
                            SetId(ref newUri, uri, uriStyle);                               //设置newUri的id
                            this.EnqueueUri(newUri, true);
                            temp = newUri.AbsoluteUri;
                        }
                        catch (Exception)
                        {
                        }
                    }
                }
            }
            catch (Exception e)
            {
            }
            finally
            {
            }
        }
        public void SetUriCanSave(ref MyUri uri)
        {
            if (_State == true)
                uri.CanSave = true;
            else
                uri.CanSave = false;
        }
        /// <summary>
        /// 设置MyUri的id
        /// </summary>
        /// <param name="newUri"></param>
        /// <param name="uriStyle"></param>
        private void SetId(ref MyUri newUri, MyUri parentUri, int uriStyle)
        {
            switch (uriStyle)
            {
                case 1:
                    newUri.Fk_seed = parentUri.Id;
                    break;
                case 2:
                    newUri.Fk_forum = parentUri.Id;
                    newUri.Fk_seed = seedUri.Id;
                    break;
                default:
                    newUri.Fk_seed = 0;
                    newUri.Fk_forum = 0;
                    break;
            }
        }
        /// <summary>
        /// 根据字符串解析页面
        /// </summary>
        /// <param name="strResponse"></param>
        private MatchCollection GetRefs(string strResponse, ref string strStatus)
        {
            string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(strResponse);
            strStatus += "Found: " + matches.Count + " ref(s)\r\n";
            return matches;
        }
        /// <summary>
        /// 获取新的MyUri
        /// </summary>
        /// <param name="strRef"></param>
        /// <param name="uri"></param>
        /// <returns></returns>
        private MyUri GetNewUri(string strRef, MyUri uri)
        {
            if (strRef.IndexOf("..") != -1 || strRef.StartsWith("/") == true || strRef.StartsWith("http://") == false)
                strRef = new Uri(uri, strRef).AbsoluteUri;
            Normalize(ref strRef);
            MyUri newUri = new MyUri(strRef);
            if (newUri.Scheme != Uri.UriSchemeHttp && newUri.Scheme != Uri.UriSchemeHttps)
                return null;
            if (newUri.Host != uri.Host)
                return null;
            newUri.Depth = uri.Depth + 1;
            return newUri;
        }
        /// <summary>
        /// 标准化url
        /// </summary>
        /// <param name="strURL"></param>
        private void Normalize(ref string strURL)
        {
            if (strURL.StartsWith("http://") == false)
                strURL = "http://" + strURL;
            if (strURL.IndexOf("/", 8) == -1)
                strURL += '/';
        }
        /// <summary>
        /// 判断种子uri是否过期，过期的话修改IsStartFromSeed为true
        /// </summary>
        /// <param name="uri"></param>
        /// <returns></returns>
        private bool IsSeedTimeOut(MyUri uri)
        {
            return SeedHelper.IsSeedTimeout(uri);
        }
        /// <summary>
        /// 判断种子uri在表中是否存在
        /// </summary>
        /// <param name="uri"></param>
        /// <returns></returns>
        private bool IsExistSeed(MyUri uri)
        {
            return SeedHelper.IsSeedExist(uri);
        }
    }
}